/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package crp.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Scanner;
import crp.object.Keywords;

import edu.stanford.nlp.process.Morphology;


/**
 *
 * @author ouhang
 */
public class Utility {
    public static int frequencyParam = 0;
    public static double cutoff = 0.0;
    public static HashSet<String> trivialWords = new HashSet<String>();
  
    /** Document encoding */
	public static String documentEncoding = "default";
    
    public static Hashtable<String,Integer> getWordFreqOfArticles(File[] articles, HashSet<String> trivialWords) throws FileNotFoundException{
        Hashtable<String, Integer> wt = new Hashtable<String,Integer>();
        Morphology m = new Morphology();
        for(File article:articles){
            Scanner scanner = new Scanner(article);
            while(scanner.hasNextLine()){
                String line = scanner.nextLine();
                line = line.toLowerCase();
                String[] words = line.split("[^a-zA-Z]+");
                for(String w: words){
                	
                	line.replaceAll("[\\p{Punct}+&&[^-]]", " ");
                    if(trivialWords.contains(w)) continue;
                    
                    w = m.stem(w);
                    
                    if(w.length() < 3) continue;
                    
                    if(trivialWords.contains(w)) continue;
            
                    if(wt.containsKey(w)){
                            int count = wt.get(w);
                            wt.put(w,++count);
                    }
                    else wt.put(w,1);
                }
            }
            scanner.close();
        }
        
        if(wt.containsKey("")) wt.remove("");
        for(Iterator<String> it = wt.keySet().iterator() ; it.hasNext();){
        	String key  = it.next();
        	if(wt.get(key) < Utility.frequencyParam)it.remove();
        }
        return wt;
    }
    
    
    public static Hashtable<String,Integer> getWordFreqOfOneArticle(File article, HashSet<String> trivialWords) throws FileNotFoundException{
    	Hashtable<String, Integer> wt = new Hashtable<String,Integer>();
    	Scanner scanner = new Scanner(article);

    	Morphology m = new Morphology();
    	
    	ArrayList<String> keywords = Keywords.getKeywords();
        
    	for(String word:keywords) wt.put(word,0);
    	
    	while(scanner.hasNextLine()){
            String line = scanner.nextLine();
            line = line.toLowerCase();
            String[] words = line.split("[^a-zA-Z]+");
            for(String w: words){
            	line.replaceAll("[\\p{Punct}+&&[^-]]", " ");
                if(trivialWords.contains(w)) continue;

                w = m.stem(w);
                
                if(w.length() < 3) continue;
                
                if(trivialWords.contains(w)) continue;
        
                if(wt.containsKey(w)){
                        int count = wt.get(w);
                        wt.put(w,++count);
                }
                else wt.put(w,1);
            }
        }
        scanner.close();
        return wt;
    }
    
    public static double[] getWordFreqOfOneArticle2(File article, HashSet<String> trivialWords) throws FileNotFoundException{

    	double[] wt = new double[Keywords.getKeywords().size()+1];//the last position of this vector is the total word number in the article
    	Scanner scanner = new Scanner(article);

    	Morphology m = new Morphology();
        
    	for(int i = 0 ; i < wt.length ; i++) wt[i] = 0;
    	
    	while(scanner.hasNextLine()){
            String line = scanner.nextLine();
            line = line.toLowerCase();
            //line.replaceAll("[\\p{Punct}+&&[^-]]", " ");
            String[] words = line.split("[^a-zA-Z]+");
            for(String w: words){
            	wt[wt.length-1]++;
                if(trivialWords.contains(w)) continue;

                w = m.stem(w);
                
                if(w.length() < 3) continue;
                
                if(trivialWords.contains(w)) continue;
        
                for(int i = 0 ; i < Keywords.getKeywords().size() ; i++)
                	if(Keywords.getKeywords().get(i).equals(w)) wt[i]++;
            }
        }
        scanner.close();
        return wt;
    }
    
    public static double[] getWordFreqOfOneArticle3(String text){
    	//text.replaceAll("[\\p{Punct}+&&[^-]]", " ");
    	double[] wt = new double[Keywords.getKeywords().size()+1];
        String[] words = text.split("\n");
        for(String w: words){
        	wt[wt.length-1]++;
        	for(int i = 0 ; i < Keywords.getKeywords().size() ; i++)
            	if(Keywords.getKeywords().get(i).equals(w)) wt[i]++;
        }
        return wt;
    }
    
   
    public static String[] fileToStringArray (File f, String encoding)
	{
		ArrayList<String> wordarray = new ArrayList<String>();

		try {

			BufferedReader input = null;
			if (encoding == null) {
				input = new BufferedReader (new FileReader (f));
			}
			else {
				input = new BufferedReader( new InputStreamReader( new FileInputStream(f), encoding ));
			}
			String line;

			while (( line = input.readLine()) != null) {
				String[] words = line.split ("\\s+");
				for (int i = 0; i < words.length; i++)
					wordarray.add (words[i]);
			}

		} catch (IOException e) {
			throw new IllegalArgumentException("Trouble reading file "+f);
		}
		return (String[]) wordarray.toArray(new String[]{});
	}
    
    
    public static int getUpperTriangIndex(int i, int j, int dim){
    	if(i > j) return -1;
    	return dim * i - (i+1) * i / 2 + j;
    }
    
    public static void loadTrivialWords(String path) throws FileNotFoundException{
    	File triwfile = new File(path);
    	Scanner scanner = new Scanner(triwfile);
        
        while(scanner.hasNextLine()){
        	String line = scanner.nextLine();
            line = line.toLowerCase();
            String[] words = line.split("\\s+");
            for(String w:words)
            	Utility.trivialWords.add(w);
        }
        scanner.close();
    }
    
    public static void updateTrivialWords(String path) throws FileNotFoundException{
    	if(Utility.trivialWords == null) return;
    	Utility.trivialWords.clear();
    	Utility.loadTrivialWords(path);
    }
    
    public static double calculateAssociValue(int[] v1 , int[] v2){
    	double result = 0.0;
    	if(v1.length != v2.length) return -1;
    	for(int i = 0 ; i < v1.length ; i++){
    		result += Math.pow((v1[i]-v2[i]),2);
    	}
    	result = Math.sqrt(result/v1.length);
    	//if(result == 0.0) return -1;
    	if(result <= 2.72) return 1.0;
    	return 1/Math.log(result);
    }
    
    public static int[][] calculateIncidenceMatrix(double[][] associM){
    	int[][] incidenM = new int[associM.length][associM.length];
    	for(int i = 0 ; i < associM.length ; i++){
    		for(int j = 0 ; j < i ; j++){
    			if(associM[i][j] < Utility.cutoff){ 
    				incidenM[i][j] = 0;
    				incidenM[j][i] = 0;                                            
    			}else{
    				incidenM[i][j] = 1;
    				incidenM[j][i] = 1;   
    			}
    		}
    	}
    	return incidenM;
    }
    
    public static void main(String args[]) {
    	String filename = "D:/Test Word.txt";
    	HashSet<String> hs = new HashSet<String>();
    	hs.add("check");
    	Hashtable<String, Integer> ht = null ;
    	try {
			ht = Utility.getWordFreqOfOneArticle(new File(filename),hs);
			for(String w:ht.keySet()){
				System.out.println(w);
			}
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
    	System.out.println(ht.size());
    }
}
